import datetime

import requests
from scrapy import Selector
from urllib import parse

from csdn_spider.models import Topic


# 文章列表页数据爬取和清洗，单页逻辑，返回一个迭代器
def topic_spider(url):
    res = requests.get(url).text
    sel = Selector(text=res)
    all_trs = sel.xpath("//table[@class='forums_tab_table']//tbody//tr")
    topic = {}
    for item in all_trs:
        # 路径注意取相对路径
        topic['status'] = item.xpath(".//td[@class='forums_topic_flag']/span/text()").extract()[0]
        topic['score'] = item.xpath(".//td[@class='forums_score']/em/text()").extract()[0]
        topic['topic_url'] = item.xpath(".//td[@class='forums_topic']/a[contains(@class,'forums_title')]/@href").extract()[
            0]
        topic['topic_id'] = topic['topic_url'].split('/')[-1]
        topic['topic_title'] = item.xpath(".//td[@class='forums_topic']/a/text()").extract()[0]
        topic['author_info'] = item.xpath(".//td[@class='forums_author']/a/@href").extract()[0]
        topic['author_id'] = topic['author_info'].split('/')[-1]
        create_time = item.xpath(".//td[@class='forums_author']/em/text()").extract()[0]
        topic['create_time'] = datetime.datetime.strptime(create_time, "%Y-%m-%d %H:%M")
        reply_info = item.xpath(".//td[@class='forums_reply']/span/text()").extract()[0]
        topic['answer_nums'] = reply_info.split('/')[0]
        topic['click_nums'] = reply_info.split('/')[1]
        yield topic

    try:
        next_page = sel.xpath("//table[@class='forums_tab_table']//tfoot//a[contains(@class,'next_page')]/@href").extract()[0]
    except BaseException as e:
        next_page=False
    if next_page and next_page != "javascript:;":
        # parse.urljoin域名处理的方法，自行判断域名情况进行拼接
        next_page = parse.urljoin(url, next_page)
        for item in topic_spider(next_page):
            yield item


# 文章列表数据存入数据库
def save_topic_mysql(base_topic):
    topic = Topic()
    topic.title = base_topic['topic_title']
    topic.id = int(base_topic['topic_id'])
    topic.author_id = base_topic['author_id']
    topic.create_time = base_topic['create_time']
    topic.answer_nums = base_topic['answer_nums']
    topic.click_nums = base_topic['click_nums']
    topic.score = base_topic['score']
    topic.status = base_topic['status']
    exited_topic = Topic.select().where(Topic.id == int(base_topic['topic_id']))
    if exited_topic:
        topic.save()
    else:
        topic.save(force_insert=True)